{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "29c68421",
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch\n",
    "\n",
    "torch.manual_seed(9527)\n",
    "\n",
    "if torch.accelerator.is_available():\n",
    "    device = torch.accelerator.current_accelerator()\n",
    "else:\n",
    "    device = torch.device(\"cpu\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "81d2c279",
   "metadata": {},
   "outputs": [],
   "source": [
    "from time import perf_counter, ctime\n",
    "from datetime import timedelta\n",
    "from dataset import sos, eos\n",
    "\n",
    "\n",
    "def train(model, dataloader, backward, epochs=20):\n",
    "    losses, *_ = [], model.to(device).train()\n",
    "    label, start_time = model.__class__.__name__, perf_counter()\n",
    "    print(f\"{label} @ {ctime()}\\n\")\n",
    "    for epoch in range(epochs):\n",
    "        total_loss, start = 0, perf_counter()\n",
    "        for source, target in dataloader:\n",
    "            source, target = source.to(device), target.to(device)\n",
    "            logits = model(source, target[target != eos].view(target.size(0), -1))\n",
    "            loss = backward(logits.view(-1, logits.size(-1)), target[:, 1:].reshape(-1))\n",
    "            loss = loss.cpu().item()\n",
    "            losses.append(loss)\n",
    "            total_loss += loss\n",
    "        print(\n",
    "            f\"Epoch {epoch+1:>2}/{epochs}, Time {timedelta(seconds=int(perf_counter()-start))}, Loss {total_loss / len(dataloader)}\"\n",
    "        )\n",
    "    print(f\"\\n{label} # {timedelta(seconds=int(perf_counter() - start_time))}\\n\")\n",
    "    return losses\n",
    "\n",
    "\n",
    "@torch.no_grad()\n",
    "def eval(model, dataloader, print_result, max_len):\n",
    "    model.to(device).eval()\n",
    "    for x, Y in dataloader:\n",
    "        x, y = x.to(device), torch.full((1, 1), sos, dtype=int, device=device)\n",
    "        for _ in range(max_len - 1):\n",
    "            logits = model(x, y).argmax(dim=-1)\n",
    "            y = torch.concat([y, logits[:, -1:]], dim=-1)\n",
    "            if y[0, -1].item() == eos:\n",
    "                break\n",
    "        print_result(\n",
    "            [index - eos - 1 for index in x.view(-1).tolist() if index > eos],\n",
    "            [index - eos - 1 for index in Y.view(-1).tolist() if index > eos],\n",
    "            [index - eos - 1 for index in y.view(-1).tolist() if index > eos],\n",
    "        )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "66b0f285",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Transformer(\n",
       "  (encoder): Encoder(\n",
       "    (0-5): 6 x EncoderBlock(\n",
       "      (attn): AddAndNorm(\n",
       "        (norm): LayerNorm((512,), eps=1e-05, elementwise_affine=True)\n",
       "        (out): Dropout(p=0.1, inplace=False)\n",
       "        (sub): MultiHeadAttention(\n",
       "          (0-3): 4 x Linear(in_features=512, out_features=512, bias=True)\n",
       "        )\n",
       "      )\n",
       "      (feed): AddAndNorm(\n",
       "        (norm): LayerNorm((512,), eps=1e-05, elementwise_affine=True)\n",
       "        (out): Dropout(p=0.1, inplace=False)\n",
       "        (sub): FeedForward(\n",
       "          (0): Linear(in_features=512, out_features=2048, bias=True)\n",
       "          (1): ReLU()\n",
       "          (2): Linear(in_features=2048, out_features=512, bias=True)\n",
       "        )\n",
       "      )\n",
       "    )\n",
       "  )\n",
       "  (decoder): Decoder(\n",
       "    (0-5): 6 x DecoderBlock(\n",
       "      (att1): AddAndNorm(\n",
       "        (norm): LayerNorm((512,), eps=1e-05, elementwise_affine=True)\n",
       "        (out): Dropout(p=0.1, inplace=False)\n",
       "        (sub): MultiHeadAttention(\n",
       "          (0-3): 4 x Linear(in_features=512, out_features=512, bias=True)\n",
       "        )\n",
       "      )\n",
       "      (att2): AddAndNorm(\n",
       "        (norm): LayerNorm((512,), eps=1e-05, elementwise_affine=True)\n",
       "        (out): Dropout(p=0.1, inplace=False)\n",
       "        (sub): MultiHeadAttention(\n",
       "          (0-3): 4 x Linear(in_features=512, out_features=512, bias=True)\n",
       "        )\n",
       "      )\n",
       "      (feed): AddAndNorm(\n",
       "        (norm): LayerNorm((512,), eps=1e-05, elementwise_affine=True)\n",
       "        (out): Dropout(p=0.1, inplace=False)\n",
       "        (sub): FeedForward(\n",
       "          (0): Linear(in_features=512, out_features=2048, bias=True)\n",
       "          (1): ReLU()\n",
       "          (2): Linear(in_features=2048, out_features=512, bias=True)\n",
       "        )\n",
       "      )\n",
       "    )\n",
       "  )\n",
       "  (emi): Embedding(2388, 512, padding_idx=0)\n",
       "  (emo): Embedding(1939, 512, padding_idx=0)\n",
       "  (lin): Linear(in_features=512, out_features=1939, bias=True)\n",
       "  (pos): PositionalEncoding()\n",
       "  (out): Dropout(p=0.1, inplace=False)\n",
       ")"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from dataset import get_dataset, cmn_words, eng_words, seq_len, pad\n",
    "from model import Transformer\n",
    "\n",
    "\n",
    "train_loader, test_loader = get_dataset()\n",
    "input_size, output_size = len(cmn_words) + eos + 1, len(eng_words) + eos + 1\n",
    "\n",
    "model = Transformer(input_size, output_size, max_len=seq_len, padding_idx=pad)\n",
    "model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "d1cbaa53",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Transformer @ Thu Sep 11 19:42:40 2025\n",
      "\n",
      "Epoch  1/20, Time 0:00:36, Loss 2.3916522305399113\n",
      "Epoch  2/20, Time 0:00:36, Loss 1.3891627578495296\n",
      "Epoch  3/20, Time 0:00:36, Loss 1.0152149140425624\n",
      "Epoch  4/20, Time 0:00:36, Loss 0.7823886207093909\n",
      "Epoch  5/20, Time 0:00:35, Loss 0.6148066480454373\n",
      "Epoch  6/20, Time 0:00:34, Loss 0.49746944047766367\n",
      "Epoch  7/20, Time 0:00:35, Loss 0.4029694265372693\n",
      "Epoch  8/20, Time 0:00:35, Loss 0.33313237338644414\n",
      "Epoch  9/20, Time 0:00:34, Loss 0.28541384316827395\n",
      "Epoch 10/20, Time 0:00:35, Loss 0.24768323364328737\n",
      "Epoch 11/20, Time 0:00:35, Loss 0.2196792724373957\n",
      "Epoch 12/20, Time 0:00:35, Loss 0.19540997763383852\n",
      "Epoch 13/20, Time 0:00:35, Loss 0.18268868432846985\n",
      "Epoch 14/20, Time 0:00:35, Loss 0.16840746318516525\n",
      "Epoch 15/20, Time 0:00:36, Loss 0.15645652718722683\n",
      "Epoch 16/20, Time 0:00:35, Loss 0.14903402436754795\n",
      "Epoch 17/20, Time 0:00:35, Loss 0.14044308065686129\n",
      "Epoch 18/20, Time 0:00:34, Loss 0.13487509531512412\n",
      "Epoch 19/20, Time 0:00:34, Loss 0.12898004541783092\n",
      "Epoch 20/20, Time 0:00:34, Loss 0.12283081021713993\n",
      "\n",
      "Transformer # 0:11:48\n",
      "\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "[<matplotlib.lines.Line2D at 0x132b97c50>]"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAhYAAAGdCAYAAABO2DpVAAAAOnRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjEwLjMsIGh0dHBzOi8vbWF0cGxvdGxpYi5vcmcvZiW1igAAAAlwSFlzAAAPYQAAD2EBqD+naQAAOfBJREFUeJzt3Qd0lGX69/ErJCTUhN5DR6o06U0RBBFdcVdXXVTsq4vdtWBdC4bVd/dvxw66gigqoii9itTQa+gQek+hhJDMe647zDCTTCCTzOSZeeb7OWdOpjyZuclDMr+5y3VHOBwOhwAAAPhBCX88CQAAgCJYAAAAvyFYAAAAvyFYAAAAvyFYAAAAvyFYAAAAvyFYAAAAvyFYAAAAv4mSYpadnS179+6V8uXLS0RERHG/PAAAKAStp5mWlia1atWSEiVKBE+w0FARHx9f3C8LAAD8IDk5WerUqRM8wUJ7KpwNi42NLe6XBwAAhZCammo6Bpzv40ETLJzDHxoqCBYAAISWi01jYPImAADwG4IFAADwG4IFAADwG4IFAADwG4IFAADwG4IFAADwG4IFAADwG4IFAADwG4IFAADwG4IFAADwG4IFAADwG4IFAADwm2LfhCxQ/jMtSdJOn5UHLm8kNeJKWd0cAADCkm16LMYtTZbRC3bI0RNnrG4KAABhyzbBwskhDqubAABA2LJNsLjw7vAAAKA42CZYODnosAAAwDK2CRYRdFkAAGA52wQLAABgPdsEiwhmWQAAYDnbBAsAAGA92wULJm8CAGAd2wQLJm8CAGA92wQLJwpkAQBgHdsECzosAACwnm2ChRNzLAAAsI5tgkUEkywAALCcbYKFEx0WAABYx3bBAgAAhFCw2LNnj9x2221SuXJlKV26tFx66aWSmJgowcLBJAsAACwT5cvBx44dk+7du0vv3r1l8uTJUrVqVdm8ebNUrFhRrMYUCwAAQixY/Pvf/5b4+HgZNWqU674GDRpIMKG/AgCAEBkK+fnnn6VDhw5y0003SbVq1aRdu3by6aefXvB7MjIyJDU11eMSCPRYAAAQYsFi27ZtMnLkSGnSpIlMnTpVHnzwQXnkkUfkyy+/zPd7EhISJC4uznXRHo9AYooFAADWiXD4MNsxOjra9FgsWLDAdZ8Gi6VLl8rChQvz7bHQi5P2WGi4SElJkdjYWPGXXm/Oll1HT8oPD3aTy+pZP+cDAAA70fdv7SC42Pu3Tz0WNWvWlBYtWnjc17x5c9m1a1e+3xMTE2Ma4H4BAAD25FOw0BUhSUlJHvdt2rRJ6tWrJ8GDsRAAAEIiWDz++OOyaNEieeONN2TLli0yduxY+eSTT2To0KFiNSZvAgAQYsGiY8eOMmHCBPnmm2+kVatW8tprr8nbb78tgwcPlmDB5E0AAEKkjoW69tprzSXY0GEBAID1bLdXCB0WAABYxzbBgm3TAQCwnm2ChRNzLAAAsI5tggX9FQAAWM82wcKJbdMBALCOfYIFXRYAAFjOPsHiHPorAACwjm2CBR0WAABYzzbBwokpFgAAWMc2wYI6FgAAWM82wQIAAFjPdsHCwfRNAAAsY5tgwUAIAADWs02wcKHDAgAAy9gmWDB3EwAA69kmWDjRYQEAgHVsEywimGUBAIDlbBMsnCiQBQCAdWwTLJhjAQCA9WwTLJyoYwEAgHVsFywAAIB1bBcsmGMBAIB1bBMs2IQMAADr2SZYONFhAQCAdWwTLOivAADAerYJFgAAwHq2CxYOZm8CAGAZ2wQL5m4CAGA92wQLJ/orAACwjm2CBT0WAABYzzbBwoUuCwAALGObYMG26QAAWM82wcKJTcgAALCObYIFcywAALCebYKFE2UsAACwjm2CBR0WAABYzzbBwokeCwAArGOfYMEkCwAALGefYHEOHRYAAFjHNsGC/goAAKxnm2ABAACsZ7tgwbbpAACESLD417/+JRERER6XZs2aSTBg7iYAANaL8vUbWrZsKTNmzDj/BFE+P0VA0V8BAIB1fE4FGiRq1KghwYYOCwAAQnCOxebNm6VWrVrSsGFDGTx4sOzateuCx2dkZEhqaqrHJZCYYgEAQIgEi86dO8vo0aNlypQpMnLkSNm+fbv07NlT0tLS8v2ehIQEiYuLc13i4+MlEHS+BwAACKFgMWDAALnpppukdevW0r9/f/ntt9/k+PHj8t133+X7PcOGDZOUlBTXJTk5WQKLLgsAAKxSpJmXFSpUkEsuuUS2bNmS7zExMTHmEmj0VwAAEOJ1LNLT02Xr1q1Ss2ZNCRbMsQAAIESCxT//+U+ZO3eu7NixQxYsWCA33HCDREZGyq233ipWY4oFAAAhNhSye/duEyKOHDkiVatWlR49esiiRYvM9WBBhwUAACESLMaNGyfBKoJZFgAAWM6Ge4VY3QIAAMKXbYLFkh1Hzde9x09Z3RQAAMKWbYKF0/DfNljdBAAAwpbtggUAALAOwQIAAPiN7YLFgFbBt/MqAADhwjbB4uqWOYGia6PKVjcFAICwZZtgERmZU8ciO5v1pgAAWMU2waLEuZrexAoAAKxjo2CR85UOCwAArGObYOEs6O2g9CYAAJax31AIuQIAAMvYJlhEnAsW2SQLAAAsY6NgkfOVORYAAFjHdpM3HawLAQDAMjYKFsyxAADAavYbCmEsBAAAy9goWFAgCwAAq9mwQBbRAgAAq9gmWEScK5HFSAgAANaxXY8FszcBALCODQtkWd0SAADCl+2WmzLHAgAA69gmWFB5EwAA69kmWFB5EwAA69koWFB5EwAAq9kmWJxbbUrlTQAALGS/HgurGwIAQBizUbDI+cqqEAAArGO7ypvkCgAArGO/VSEkCwAALGObYEHlTQAArGejYJHzlTkWAABYxzbBglUhAABYz0bBIucrcywAALCO/eZYZFvdEgAAwpdtgkXKqUzz9ccVu61uCgAAYcs2weKTedvM18wshkIAALCKbYIFAACwHsECAAD4DcECAAD4DcECAAAER7AYMWKEWeb52GOP+a9FAAAg/ILF0qVL5eOPP5bWrVv7t0UAACC8gkV6eroMHjxYPv30U6lYsaL/WwUAAMInWAwdOlQGDhwoffv2veixGRkZkpqa6nEBAAD2FOXrN4wbN06WL19uhkIKIiEhQV555ZXCtA0AANi5xyI5OVkeffRRGTNmjJQqVapA3zNs2DBJSUlxXfQ5AACAPfnUY7Fs2TI5ePCgtG/f3nVfVlaWzJs3T95//30z7BEZGenxPTExMeYCAADsz6dg0adPH1mzZo3HfXfddZc0a9ZMnnnmmTyhojhVKhstR0+cMdcTdxyVDvUrWdYWAADClU/Bonz58tKqVSuP+8qWLSuVK1fOc39x69Wkivy0cq+5vu3QCYIFAAAWsE3lTS3U5ZSecdbStgAAEK58XhWS25w5cyQYuOUKyXawdToAAFawTY9FCbdkkZVNsAAAwAq2CRZuHRZCrgAAwBq27LFgKAQAAGvYJ1i4/Uuy6bIAAMAStlwVkkWPBQAAlrBNsLikWjnXdTosAACwhm2Cxa2d67quMxQCAIA1bBMsoiPP/1MYCgEAwBq2nGNBjwUAANawTbBwR4EsAACsYctg8dn87VY3AQCAsGTLYAEAAKxBsAAAAH5DsAAAAH5DsAAAAH5DsAAAAH5j22BxOjPL6iYAABB2bBssEnccs7oJAACEHdsGi68X7bS6CQAAhB3bBovlu+ixAACguNk2WKSezrS6CQAAhB3bBgu2CwEAoPjZN1iQLAAAKHa2DRZZDoIFAADFzVbB4s5u9V3XyRUAABQ/WwWLCmVKWt0EAADCmq2CBdMqAACwlq2CBeMfAABYy1bBom+L6lY3AQCAsGarYNGqVpzH7V1HTlrWFgAAwpGtgkVuvd6abXUTAAAIK7YKFhERVrcAAIDwZrNgQbIAAMBKtgoWAADAWgQLAADgNwQLAADgNwQLAADgNwQLAADgNwQLAADgN7YPFtnsTAYAQLGxfbAAAADFx/bB4vTZLKubAABA2PApWIwcOVJat24tsbGx5tK1a1eZPHmyBLOP5m6zugkAAIQNn4JFnTp1ZMSIEbJs2TJJTEyUK6+8Uq6//npZt26dBKtdR05Y3QQAAMJGlC8HX3fddR63hw8fbnoxFi1aJC1btpRgxNxNAACCNFi4y8rKkvHjx8uJEyfMkEh+MjIyzMUpNTVVilOWg2QBAEDQTt5cs2aNlCtXTmJiYuSBBx6QCRMmSIsWLfI9PiEhQeLi4lyX+Ph4KU4OggUAAMEbLJo2bSorV66UxYsXy4MPPihDhgyR9evX53v8sGHDJCUlxXVJTk6WQCobHelxO+VUZkBfDwAAFCFYREdHS+PGjeWyyy4zvRFt2rSRd955J9/jtWfDuYrEeQmkPs2re9z+Y8sROZh6OqCvCQAA/FTHIjs722MOhdVKROS9b+bGg1Y0BQCAsOPT5E0d1hgwYIDUrVtX0tLSZOzYsTJnzhyZOnWqBLOMTIpkAQAQdMHi4MGDcscdd8i+ffvMREwtlqWh4qqrrpJglnE22+omAAAQFnwKFp9//rkEO29rQM5SzAIAgGJhu71CasaVtroJAACELdsFi4evbGx1EwAACFu2CxZlY6IkrnRJj/vempok8zYdsqxNAACEC9sFC1W5bHSe++74YoklbQEAIJzYMliU8FbMAgAABJwtg0UUwQIAAEvYMlhEEiwAALCELYPFkG71rW4CAABhyZbB4sb2daxuAgAAYSmsJm9mU4ETAICAsmWwyE/nhJnicBAuAAAIlLAKFofSMmQOhbIAAAiYsAoWasyiXVY3AQAA2wq7YDFjwwGrmwAAgG3ZNlg0qFI238eSj54s1rYAABAubBssLqtXMd/Her45Ww6mni7W9gAAEA5sGywuVnvzlUnri6klAACED9sGi/hKZS74+K+r9xVbWwAACBe2DRb9Wla3ugkAAIQd2waLptXLW90EAADCjm2DRUQEO5wCAFDcbBssAABA8YuSMDbkiyXSvm5FaVytnAxsXdPq5gAAEPLCOljM3XTIXNTA1gOtbg4AACGPoRAAAOA3tg4Wr/yppdVNAAAgrNg6WAzpVl9u7RRvdTMAAAgbtg4WKiYq0uomAAAQNmwfLBwOR4GOy84u2HEAACCcg0UBj1u9J0Uys7ID3BoAAOzN9sGioAZ98Ic8/u1Kq5sBAEBII1i4mcSOpwAAFIntg0UBp1gAAAA/sH+wKPAsCwAAUFS2Dxa+uu+rRNmXcsrqZgAAEJJsHyx8HQqZvv6APPPDmkA1BwAAW7N9sCgX4/s+a/uO02MBAEBh2D5Y/OOKxtKhXkWpWj7G6qYAAGB7tg8WcWVKyvcPdpPbOtezuikAANie7YOFU4kI344/duJMoJoCAIBthU2wiPAhWGw+mC7tXpsu4xOTA9kkAABsJ4yChY9dFiLy4sS1AWkLAAB25VOwSEhIkI4dO0r58uWlWrVqMmjQIElKShK7ymZPMgAAAhcs5s6dK0OHDpVFixbJ9OnTJTMzU/r16ycnTpyQYFeIDgs5w26nAAD4xKciD1OmTPG4PXr0aNNzsWzZMunVq5cEs95Nq8mbU+zbuwIAQMjPsUhJSTFfK1WqlO8xGRkZkpqa6nGxQvOasTLzyculR+Mqlrw+AADhoNDBIjs7Wx577DHp3r27tGrV6oLzMuLi4lyX+Ph4sUqjquWkUtlon75n04G0gLUHAAC7KXSw0LkWa9eulXHjxl3wuGHDhpmeDeclOdnaJZx/bl/bp+P7/d88ueWTheJg/3UAAAITLB566CGZNGmSzJ49W+rUqXPBY2NiYiQ2NtbjYqUrmlaTMfd29ul7Fm07KlsPpcu/p2yUFbuOBaxtAACEVbDQT+0aKiZMmCCzZs2SBg0aSCjqXoh5Fu/P2iIj52yVGz5cEJA2AQAQdqtCdPhj7NixMnHiRFPLYv/+/eZ+nTtRunRpsbOkA+lWNwEAAHv1WIwcOdLMk7jiiiukZs2arsu3334buBYCAAB79ljYaQLj9W1rycSVewt8/IZ91iyTBQAglITNXiG5vX1z20J/7+ykg3L/V4lyOD3Dr20CACCseizCfVMyp7tGLTVfS0dHyju3tPNjqwAACG1h22PhD/tTTlvdBAAAgkrY9liolS9dJftTT8uKXcdl2I9rrG4OAAAhL6x7LCqUiZZmNWLlhna+VeN02nwwXY6kZ8h3icly8sxZv7cPAIBQE9Y9FkV19MQZGfzZYtm4P00SdxyVN29sY3WTAACwVFj3WDiVKMJETg0VavKanGJhAACEM4KFWSFS9OdIyyjYUMi6vSkydR0hBABgTwyFaLAoxtca+O588/Xnh7pL6zoVivGVAQAIPHosNF1FlpD+LasX+XkyzmYV+NgtB9l7BABgPwSLcz6+vYO8MLB5kZ7jts8WF7js+csT18mZs9lFej0AAIINwcJNh/qVivT9S3cck05vzJS9x0+Z2xoynvl+tXwwe4vXORljFu8s0usBABBsCBZu2sYXfc7DobQMeXvGJnN9RfJx+TYxWd6amuT1WC3OBQCAnRAsAmD17hTZc/yUnDpzfs7FH1sO22p3WAAAvCFYBIDWtug+YpbHfVpIa/r6A5a1CQCA4kCwyKV2hdJ+e67cq0Tmbjrkt+cGACAYESxyefX6ln57rrtHJ/rtuQAACAUEi1zKlyppdRMAAAhZBItcWtSKDdhz/7h8T8CeGwCAYECwyKVcTJSsfaW/JL1+td+f+1RmwStzAgAQiggW+YSLmKhIv2xOdiEfz90mU9buC+yLAABQjAgWF9C0evmAv8YDXy8P+GsAAFBcCBZB4nRmlvzr53UyjyWpAIAQRrAIAok7jsqoP3bI6AU75I4vlljdHAAACo1gcQHFVYF72c5jsvvYyeJ5MQAAAohgcQGBnrxp1WsBABAoBIsLKM49wyKEZAEACH0Eiwu4v1fDYnsteiwAAHZAsLiANvEViu21CpIr/t/UJHnkmxVyNiu7GFoEAIDvCBZB7mDaaXE4HDL6j+3y/uwt8vOqvTJ57X6rmwUAgFdR3u9GcQ5P6FSOrYdO5Ll/+voDct9XiTKobS35aeVe1/3pGWdd14+kZ8inv2+XmzvGS4MqZYunwQAA5INgcQFVysUUy+v8Z1qSZGblnSn63qzN5qt7qMg9qfSf41fJ7KRD8r+FO2Tdq/7f3wQAAF8wFHIBcaVLysSh3eXhKxu77uvdtKrfX8dbqFCrd6d4vd9h+jhyLN913Hw9cYYNzgAA1iNYFGAC5z09GpjrTaqVk1a144plXsU7M3J6K7x5fsJa2Xv8lIz6Y7uknMoMeHsAACgohkIKoEKZaFn3Sn+JiSohJzOz5L1ZWwL6ehmZ2fJ/MzZd8JhuI2bluU93Su3aqIrpaQEAwAoEiwIqG5Pzo4qNDHwnz5cLdhRpp9QdIwb6uUUAABQMQyFFVK9yGb8/52fzt/v9OQEAKA4EiyKa/eQVEmx2HTkpE1fukezsYqxJDgAAQyFFV6JE8NXi7vXWbPM142y2/LVDvNXNAQCEEXos/OCKplUlCPOFLN521OomAADCDD0WfjDqzo6mFsWmA2mmKuYtnyySYOBe70KdzsyS5TuPyd6U07J693H513Utg7LHBQAQuggWfhARESHRURGmxsUpL4Wq/tK+jvywfLfl274/OX6V/Lp6n+t2l4aV5ZpLaxZ7uwAA9uXzUMi8efPkuuuuk1q1apk31J9++ikwLbOR8qWsyW+6eZk791Chjp44U8wtAgDYnc/B4sSJE9KmTRv54IMPAtOiEJd7+EHVjCslobDJmg6VJO1Ps6o5AAAb8Pmj9IABA8wFF/fOLW0lJiry3P6lOfo2ryYtasbK3M2HZVVyzj4fgXKxxaYRcj5ZHD95Rtq+Ot1cv/ySqvLl3Z0C2jYAgD0FvI8+IyPDXJxSU1Ml1JWLifLYutxd6ZKR0ql+JTmVmSXXta5lJkdOXnN+COKj2y6TqMgS8kS/plL/2V8D2k733of9KafzPU4nnf7t08Wu23M3HQpouwAA9hXw5aYJCQkSFxfnusTHh35dhf/d08lsSPaVl0/1Ou/k2793MbuiOldcuPccaKgoLhv3p8kPy3ZLrzdny7XvzffSVpEj6RnS7//myeH08+FP3f75+aABAEBBBfxdbtiwYZKSkuK6JCcnS6hrV7eiTH/icul1ifct1DVcuC/jzL06ozjpSpBdR0/mCQ5KW7jjyAmv3/f75sPm63szN8vAd3+XtNPsogoACIKhkJiYGHMJZ94mdAaD9ftSpVG1cvk+fiD1tPxnes4uq69NWi+dGlSWv7SvbYITAADeUMeiGFQtF5zB6quFO6V+5bL5Pt75jZmu698l7jaXstGRMoDaFwAAfwWL9PR02bJli+v29u3bZeXKlVKpUiWpW7eur08XFjo1qCRP9W8qjS/QO2CVVyet9+n4tXtT5OpWNei1AAB4FeHIXUXpIubMmSO9e/fOc/+QIUNk9OjRF/1+XRWikzh1vkVsbKyEs0CvCgmkxBf6ShU/9cSMXbxLNuxLlVevb0lgAYAgVdD3b597LK644oo8FR0Rfp78bpVfal3M23RInpuwxlzv26K6qaEBAAhd7G6KQsmvjoev7vhiies6K08AIPQRLFAovgxYaA9XwuQN8suqveb2B7O3SNeEmTJhxe58K4ECAEITq0Is9Nw1zeSN3zZKKPJlKsScpEPy8dxt5vqKXcfliz+2m+uPf7uq0M8JAAhO9FhY6P5ejXw6fvwDXSVYeOtdOJuVLX96f75c/fY8Wb7rmOv+v3+9zHXdGSq82XeBsuMAgNBAsAghHetXyvcx3disOHnrXUjceUxW704xpcT//OECeXvGJunw+gw5cza7QM/5xm8b/N9QAECxIlgEmQ71KsqVzaq5bl9aOy7PJme5adEqrS2hKpQpKT8N7V7swWLP8VNyyyeLPO57e8Zmr6XE85OVXbjVRgfTTsvQMctlwdacMuQAAOswxyJI9GxSRV67vpXUqlBadh87abZUv79XQ+ndrJrc/1WiPNKniTnu54e6y+gFO2Ro78bSbcQsc5++HT94RSNTgKtzg0oSVSLweXHRtqMet/8zLUms8sKEtTJt/QH5dc0+2TFioGXtAAAQLIKG9kzUr5JTXrth1XKmAJWzWNScp84XJGtSvbwMv+FSc12DxyfztsnzA5tLycgSck0xl9o+eeaszNhwULYfOiE/Lt8jVtl97JRlrw0A8ESwsNjkR3vKjPUH5N6eDT3uL0gFymEDmsk9PRpI9dhSYoUWL031+3NOWbtPrm5VM99lq9pbs2ZPijxweSOZuna/3Nalnt/bAAAoPIKFxZrXjDWXwtDwYVWoCJQHvl6e73DGK7+sN8FCOXtIVu0+7nGMzrPo1qhKMbQUAOANkzcRMpyhwp0Oxej2705/+3Sx6/oPy3ab/Vj0oktgdTksACCwCBYIavM3H5ZuCTNlTtLBAn+Plga/98tEeXL8+QJcugR2+S7P3g0AgP8xFIKgk3IyU9IyMmXwZ4tl55GT5r47Ry0t8Pc//u1K05OR29R1+80W9gCAwKHHAkGnzavTTLEsZ6jwlbdQoU5cZOM03Vjtmnd+N4W9vNGhFN3ifdG2I4VqFwCEA3oswsid3erLswOaybszN8uHc7b69L3NapQ3wwnF5bc1+/3+nG3jK1zw8TGLdpr5Gnp5rO8lrvt1W/e40iWlQumSkjA5Z2+XSQ/3kCe+Wyk140rL50M6SFQkGR0AFMEijPRrWV1KlYws0GZf9/ZoIJ/N326qgPZoXMXUyOiSMFNCWX6FPTOzsuWz37fLyuTz+5s4zdt0yPRS5Hbd+/PF4RDZdCBdfli+W27uWDcQTQaAkEOwgEx/vJep3vnINytcvRJadGtwl3pSv3IZV02Nda/0l5Yv+792RXHJ1iSQT0/Fv6d432X2ji+WeL3f/akOpBa8bDkA2B39tzZ142V18txXuWxMnv1Hbu1U11TzvKR6efn0jg5ybeua8stDPUyYaFClrEehrrIxoZ1D8wsW3oZ4xizeaQpyFQS7vQPAeaH9ToF8vXVja7mjaz1THnz+5kOy9/hpaVqjvHmsf8sa0q9FddmwP1We7t/U9T3xlcrI+39rL3Z1Nst7UNjgVgfD6fkJa2X4rwXbbXWZ2xbxvrcpWyau3GtWq+jPHwBCHcHCprSnoXWdnMmKuUtk62Of3NGhUM/798sbysdzt0koOpvtvUDWqt0pXu8/eSarQM87J+mQz205dSZLFm0/Ikn702TEuQmhWnFUg8a4pcnSolastK9b0efnBQCrESzgk2EDmodssNDCm8t2HpX4imVk+oYDplfCKk+OX5ln5cu9Xy71WCo79t7O0q0x5ckBhBaCBcJGfhM0i4vW0fh2abIMbF3T63La3PU3/vbZ4nz3Tck4myUxUZEBaysAFBaTN+Gzb+7rIlc0rVqgY/9zUxuzE2mfZtXkhna1PR57pE8TeW1QK5n3VG+z+iSUaU0L98mev6zaKwu3HnEtZ1X3fLlUXp20Xjq/UfBlu972N/lywQ5p+sIUmbZuv0fQuHHkAkmYXLB5Ib46czZbVuw6Jln5rdkFgHPosYDPujaqbC66uZfqUK+iHDt5RrYeOuE6ZuNrV5tVGGWiozzmFRw5cUYWbT0ifZpXkwcub+h6/J/9m8pDY1dIqNLdVm9sX0fmbzksdSqWMUW11NLn+8rlb802K3EWbz/q8/N++vt2efCKRq5AobvZvvzzOnP74W9WSNLrA8z1KWv3S+LOY+ZyTauasnpPivRuWlWSj56Szg0qyeS1+2X3sZPm+wflCngF8fh3K+XX1fvksb5NPIqHAUBuBAsU2tDejeSnFXvNRFCtTNnoud9cj2khrtxKR0fKV3d3ErvSoYvcOg6fYb4WJlSo6ev3m2ChkzydgcIp42y29Hxzlrx/a3vJyDzfs3H9B394tqtzXY8iXxrqypcq6VM7NFSot2dslps7xpuKowDgDUMhKLSn+jeT+c/0lkployWyBNUcAsE58HAk3XsRLu2R0CBxIPV0vs+Ru3JofstuC6prwqwC1/gAEH4IFigS9wJaq17qJz2bVJEfHuzm8/NE5lNnXOttXMwl1cuZoRed6Ni0ek6tDvXKn1pKqNPho9w/Z2/+M937xmnevDk1KU8wWL7rmCQfPenTDrI67+J0ZsGW5AIIHwyFwG/iypSU/93TuVDfm98b521d6sm09Qcu+L1f39vZNfTy7DXN5K5RS82GaxXLRkuo06qgu46clFs/XeS35/xmyS7Zdihdvv17Vzl64oz8uHy3vH6uGNjaV/qbSaBt6lSQ2hVLy187xEuNuFJ5nuOnlXvNRdWtVEY61q8kvS6pYoqv5R4G0wCip7fkBTZq+33zITmb7ZCuDSt7HUYLVjq5Vs9Ri5qxUoJeO8AgWCBI5O1a194PvXjzzi1t5eeVe6V9vYpSrfz5N77eTavJqpf7SWypKLPp2OJtR2SMl03EQkmvt2b7/Tmdcz50m/j9bsMo45bsMm+UzjLn/52+Kd9z4LTr6Elz0c3Y1Ns3t3VNENVVJJe8MNlcX/ZCXzmRkSXxlUrL14t3yawNB0zY0H1pbv/8/J4sf25f26wmysxymCEe7Vyp68OqoUNpGTJ30yE5duKMXN+2llSLLWWeR1+rbEykjP5jh1zRtJqpRLtmd4ocPpFh/t9cjPbyaABeuuOoqeXy8nUt5L1Zm+W7xN1mhdMTVwV2Uuv2wyfk5JmzJsRcrAcr2OjKqAsFy2CkgVh78rRQXXRUaLXdahGOYh4sTU1Nlbi4OElJSZHY2NjifGkEsanr9svf/7fMXN88fIDHHyHn6hOnq1pUN/uaFFTu7/dVl4aVZNG2wk2+DGY6dFTUn01+tOcj86yWK98j//plfZGfb9RdHaV7oyoX/QOfejpTWv9rmsd9kx/tKQPe+d1c15Uys89VSnX/98/+5xVmb5z8aJi4/6tEefm6lvLYtyvNfW3qxHlUbf3hwa6yeneK6S3TN/6Uk5lmZZQ/es501c8DX+f8fpQuGSkbXrtaQsWoP7bLK7+sl//d00l6NinYMvVg8M/xq+T7ZbvN5Oc3brjU6uYEhYK+fxPDEBS0LkazGuXl5g7xeT7Z6C+2u9oVfFuRMKhtrSK17fVBl5o3oRUvXuW6r3psjPw0tLuEsv8t3BGw5/5k3ja55ZNFfgkVSoe3tA5I7k+UC7YcNvM89PPRVwt3yL9yrZxRzlChnKFCNRx2PlTtPHJ+qbQ+1/4Uz8mw936ZKMdOZrpChbdS8H8ZudC8gf68aq9MWbtP2rw6Tdq9Nt2002nG+gPy32l557iotNOZee7bcfiE+bT/+fzz1W5PZWbJte/9LgfT8p+wm1vijqOyfm/ePXG8cW+v0p/x1W/PM5/eC0N/Js55Of6mPThLth+V7ADUV9FQ4W3yc0Es2nZEthxMz/dx7Un9eO5W206CZigEQUGrSE55rJfXx179U0sz1j8o1zLKgtI5ABdTvlSUNK8Za7rO3Ut96zBAo6o5n2T1k+f/3dxG0jOy5NaO8SG/EubFiXnfhP3l+MkzknQg766xRfH75sMet9/4bYOMXrDDVDJ1Lof1hft70crk42Z4ZMHWw/K3T3OWDV9zaQ1pF19R7u7RIN+dcb15dJznG+j6fammd0N7Me79KtHc16p2nHRvXMW1Y7CGjXdnbZGRg9vLgEtz9va5Z/RSmbnRsxqr09o9qWaPmf/+te1F26MB5MaPFprruSu5aoCatn6/2eU4qkSEjJiy0QzzaC+P/j6o2z5fbH5WOvdmW4L3SrD5KUpBNX3T1SXW+vt7b8+GXo8Z8sUSWbrjmLwwsHm+x3jjnHSs83n0dXRfIOe5mJN0UD77fXuh273tULoJ1Sq/yrk3n3tc/23O820nBAsEvajIEtI2PmdDtUK5yHi0/gHVP6RO2tWsExw/HHyZVC2fs9W80w3t8m5Hj7y+WrgzIM/79PerzKRR90/VhQkVuWl9jru6N3CFCqVl1/Uy/LeiVTN949cNsulgmjzuVljs/nPDfr8+0kNa1oozoUI9OGa5ue/a9+abuSUXK8rmHizW7U2Rf09JkueuaSZNqpWX7xKTpU7F0hLrVrPkT+/Pl8+HdHT9v+6SkFMF9qVcIfP1X9fLmHu7mOvObJBfRtDeAm8TV/UNW+fwnBfhNXjokIPOlbq9Sz2Px7RHyPn/yD007Es5JUPHLJdaFUqbUKH0OPdjdFKy9l7pB5IeueYI6YTbtq9OkwiJMEN2D41dbgrIzXiil9SvXFbuHOXZM3Yx+m9w/5Bx77nw6E7Dip4P7f3U5flO2916ynyhE7qX7DhqemP172OwIVggZAxoVcP8Abi9q+cfoIu5/JKq8u7Mza7bE/7RzfRKlIuJMr+c+ofY3Z/b1zEXBB+dKBko6/Z43+W2qPT/mMpd4EwNfHe+3NezQZ77fLX5QJrr++Zt8txtt2/z8xNTdQ6IFm3THrolz/XN9/n+2JJTjt4bZ/e9Tvq9+u3fJeVUphka7NWkqow/N3yghfCe/WG17HUbUjqcnpHnTVjnjkxYscdcdFLqZfUqeuyt462n4bkf18jyXcfNxUknD+fuzdIhKb3k7jU4evKMnD5XUG7TgTTzN0UN+WKp7Dl+SnyhvUH6M7iudU154doWkvDbRtnmVoFY5/D88eyVrrCif3PevLGN63ENN2rZzmOmLVo1996vlprJz4M75/07pz8/7VVzTnbWoaB6lcuanpsv7+4ktSuUkg/nbJWHr2xywTlDgcbkTYQM/a+q48vuZcILau2eFFMdVIcz9JfbKT3jrMdtX73401r536L8P53rnJEm1cu5lnO2qp2zHbr7J3r9A6Az/tW1rWvKJD98Akd40DdNHcYp7DDhxZ5b5Z7gq933ud/IC+q2LnXNp3b3VUjP/phT/l7pcuOODSqZfWnch77G3tfZzBFx/h55M/WxXmalj7rqv3Nl87k5Drd0jJcRf2ntOk5XCPmyX09+Qxo6J+ZSt4nCwwY0k4TJF9/o8PMhHeSeL3N6NZ6+uqn844rGrp+xlv5fcy7gzvnnFWZVllbe1XkkOiwWGeE5Tyg/uvLq96evFKvevwkWQBHor0+DYedLmSsdttl6MF3evLG1GT/VTxmzNh6UdnUrSJVyOV3Q78/aLP9v2ibXWLZO6ostXVI6NahkPmU98o3/9015+MrG8t65LnfgYnTzwKtaVDOTUv1JV8+MXrDT7DH04ZwtciDVe1VZX+lGhzdeVscsL9c9e9xtfeMa01PS+Y0ZhXq9Da9ebVYk6XPohxSdnKzXtaelKJ7q31SG9j4fLNzpxow7jhQuwF1ofkdRECyAYvL3/yXK1HU5RbxqxpWShcP6FPk527823YwTu38C0fLd3kx5rKd8t3S3WRarE/q0u1h3jdWeHWdhre6NK5tPRoO97GdSWKVKljBdykX5BAuEEp3T4CwM5y/PDmhmfm/9jWABhDB9I2/24hS/Bgsdu+00fKbHH4l2r04zSx7d6Zv6vKd75yni5PT5/O0ycs5WGXd/F2lYpaw89f1qVyGrC2lft4KZZ/LCT+dXyLhvPlexTLTHZDmt2XDjRwtc3c/+oHMPdHdXAKEVLJi8CRSRLlnT5Xq6kuTJfk398pzu1USdy/60bsblb81xjcV+OLh9nlUruSsy3tOjgdzdPadgk/rPX9sUKFjoygGdj6L/rnFLd0mn+pXMcjytqaCTxbyVc7+lU115bdJ6v/5R1Dk1Xy8K7cqpQLihxwLwA1NUKfW0X7cT1+JIWw+lS5/m1T1m1uuyNR1Ldg8fvrhQtc0lz/eRkxlZUr8QM8p1GV/j53PKd+e2ZfgAeX/2FrOs00nrJuj+IErLsLd5ZZprDP6yepXMdZ2f8v2yZHnmh/MT/AAEd49F8C2ABUKQ9gj4M1QofXN3DxVKJ3/qXInChgq19Pm+ZqKbzjpfOOxKs9xWeyS+ua+Led7ChAp1ofX0+phOUnPSFYdb3rhG3vxLa9Pzoit2nC5x26FWJ8jd3LGuVMvVM+NNlXLR8tK1LUxg0aEfdxpWOtavaOo61Ig9/7N779Z2Uli6H426stnF9xkBiltRipMVFT0WAPxGl81qbYL9KackskQJ6deyuqky6VzSq/NR/thy2NxX3e0N3rl52JmsbK8l2/XPlD6mwyL5Dbdsen2AmbmvPScaZJw9M7n31tDn0smmWtNBl/fuPnZKer6Zs9FbmehIU4XRm58f6i6tasXJiuTjZijKfd8S52vp0kYNgyUjI+TLBTsKtDQwt26NKsuCrXnrSEx7vJcp+uTtsaK6oV3tIq9wQHCZ9HAP83sWMpM3P/jgA3nrrbdk//790qZNG3nvvfekU6dOfm0YAHijf7JGzt0qb05Jkmeubib/nrJRnrzqEnm4TxOP45btPGoKFunGYZfWufgfWA090ZElzJ4YzhLYTnd0rSevXt8q3+/V6p+6N8Sjfc+34aWJa/NUINUeEt3QTGsp/K1zPbOSR19Tj9MCWmPv7SzdGudUivx60U5z7OYD6fLK9S3N1vROWnnUuWvsxax86Sp5+vvVMm39AY9ljLo3j/6btCdHe9xy15QoCK07sXDb+aBzV/f6MuqPHfL8Nc2lQ/2KcsOHC6So8lsRpT1uk1bvNcu2kdf6V/sXquaPJcHi22+/lTvuuEM++ugj6dy5s7z99tsyfvx4SUpKkmrVLt4lSLAA4E8ZZ7PMXjP+pH8WtaKk9r7op76WtXzfqlznw2hFRC3apFUpr25Vw0z09ZfbP19sikj1a1FdPrmjg2nvfV8lyqC2tU1weWfmZlcg0oqSWhvl7u4NTJjROg/9W+Ztj+7xokXjtMKps1pthTIl5b6eDWXy2n1mj5LcY/i6Ikg3XHO/z/3nmHE22/QK9f3vXI/Hvrizg6nv4pycqz01retUkNoVS8tfO9TxOKe6Sqp8TEkpHZ3359fnP3Nkq1u1ywvVcdES3yfOnDWVRbWWjP5sHj5XM0ZrSszddMgUo1JLnusjnXwspOX0VP+mporour2peXrYtEKm/r8ojMGd65o6Hbnpnkaj7uxoVoHpz1t7oOILsEdS0AQLDRMdO3aU999/39zOzs6W+Ph4efjhh+XZZ5/1W8MAAPnTcs66i2a3RlXyBAT9s65LfxtXLed1H4+C0MAwc+MBE0B0RZDuCbLz6Em5e/RSub9XQ7NiyL38dkxUiXzn2ej33vLpIlPAWoefSkdHmQnISktZ14gr5bGnia8BrsPrM6Rzg0ry1T2dZNySZBPmdKhKq9xq8Tkddsuv10r/nSuSj5kt3bV9mdnZcjbLYf7N2ot1+Vuzzfwprdui83/0HfMfvRuZeUElIiLMOdCS3cNvaOXawPD3p3u73tj18Ykr90iPxlXNZGwNOFpG/PVJ610lz1e91E8iIyNML1WjquVMQNSgdXPHeDMnaO3enI3sOpzrtdLze+SElibPkjoV/R8gijVYnDlzRsqUKSPff/+9DBo0yHX/kCFD5Pjx4zJx4sQ835ORkWEu7g3TIEKwAADYgbN+jPZ2HEnPKPCOpdPXHzATk9vk2mRRA4MGNV97yQItIHUsDh8+LFlZWVK9uudMdb29caP3ymEJCQnyyiuv+PIyAACEDGcA0JL8vriqhed7qZM/h8ysEPDlpsOGDTPpxnlJTk4O9EsCAACL+NRjUaVKFYmMjJQDB3JmFzvp7Ro1anj9npiYGHMBAAD251OPRXR0tFx22WUyc+b5mbI6eVNvd+3aNRDtAwAAIcTnRa5PPPGEmazZoUMHU7tCl5ueOHFC7rrrrsC0EAAA2DdY3HzzzXLo0CF56aWXTIGstm3bypQpU/JM6AQAAOGHkt4AAOCi2IQMAAAUO4IFAADwG4IFAADwG4IFAADwG4IFAADwG4IFAADwG4IFAACwrkBWUTnLZuh6WAAAEBqc79sXK39V7MEiLS3NfI2Pjy/ulwYAAH54H9dCWUFTeVM3Ldu7d6+UL1/etYe9v5KUhhXdlp2KnsGFcxPcOD/Bi3MTvMLx3DgcDhMqatWqJSVKlAieHgttTJ06dQL2/HqCw+UkhxrOTXDj/AQvzk3wCrdzE3eBngonJm8CAAC/IVgAAAC/sU2wiImJkZdfftl8RXDh3AQ3zk/w4twEL86NBM/kTQAAYF+26bEAAADWI1gAAAC/IVgAAAC/IVgAAAC/sU2w+OCDD6R+/fpSqlQp6dy5syxZssTqJtlKQkKCdOzY0VRMrVatmgwaNEiSkpI8jjl9+rQMHTpUKleuLOXKlZO//OUvcuDAAY9jdu3aJQMHDpQyZcqY53nqqafk7NmzHsfMmTNH2rdvb2ZbN27cWEaPHl0s/0a7GDFihKlq+9hjj7nu49xYZ8+ePXLbbbeZn33p0qXl0ksvlcTERNfjOn/+pZdekpo1a5rH+/btK5s3b/Z4jqNHj8rgwYNNIaYKFSrIPffcI+np6R7HrF69Wnr27Gn+BmpFyDfffLPY/o2hKCsrS1588UVp0KCB+bk3atRIXnvtNY99MDg3heSwgXHjxjmio6MdX3zxhWPdunWO++67z1GhQgXHgQMHrG6abfTv398xatQox9q1ax0rV650XHPNNY66des60tPTXcc88MADjvj4eMfMmTMdiYmJji5duji6devmevzs2bOOVq1aOfr27etYsWKF47fffnNUqVLFMWzYMNcx27Ztc5QpU8bxxBNPONavX+947733HJGRkY4pU6YU+785FC1ZssRRv359R+vWrR2PPvqo637OjTWOHj3qqFevnuPOO+90LF682PwMp06d6tiyZYvrmBEjRjji4uIcP/30k2PVqlWOP/3pT44GDRo4Tp065Trm6quvdrRp08axaNEix++//+5o3Lix49Zbb3U9npKS4qhevbpj8ODB5nf0m2++cZQuXdrx8ccfF/u/OVQMHz7cUblyZcekSZMc27dvd4wfP95Rrlw5xzvvvOM6hnNTOLYIFp06dXIMHTrUdTsrK8tRq1YtR0JCgqXtsrODBw9qrHfMnTvX3D5+/LijZMmS5pfTacOGDeaYhQsXmtv6ZlWiRAnH/v37XceMHDnSERsb68jIyDC3n376aUfLli09Xuvmm282wQYXlpaW5mjSpIlj+vTpjssvv9wVLDg31nnmmWccPXr0yPfx7OxsR40aNRxvvfWW6z49XzExMeYNSGmI03O1dOlS1zGTJ092REREOPbs2WNuf/jhh46KFSu6zpXztZs2bRqgf1noGzhwoOPuu+/2uO/Pf/6zCQCKc1N4IT8UcubMGVm2bJnponLfj0RvL1y40NK22VlKSor5WqlSJfNVz0FmZqbHeWjWrJnUrVvXdR70q3YDV69e3XVM//79zWY+69atcx3j/hzOYziXF6dDHTqUkfvnx7mxzs8//ywdOnSQm266yQwvtWvXTj799FPX49u3b5f9+/d7/Fx1LwYdznU/N9rFrs/jpMfr37nFixe7junVq5dER0d7nBsdrjx27Fgx/WtDS7du3WTmzJmyadMmc3vVqlUyf/58GTBggLnNuSm8Yt+EzN8OHz5sxsrc/yAqvb1x40bL2mVnukOtjt93795dWrVqZe7TX0D9xdFfstznQR9zHuPtPDkfu9Ax+gZ36tQpM86JvMaNGyfLly+XpUuX5nmMc2Odbdu2yciRI+WJJ56Q5557zpyfRx55xJyPIUOGuH623n6u7j93DSXuoqKiTKh3P0bnCuR+DudjFStWDOi/MxQ9++yz5v+uhuzIyEjzPjJ8+HAzX0JxbsI4WMCaT8Zr16416R7W022bH330UZk+fbqZHIbgCuH6afaNN94wt7XHQn93PvroIxMsYJ3vvvtOxowZI2PHjpWWLVvKypUrzQcm3RKcc1M0IT8UUqVKFZM2c89w19s1atSwrF129dBDD8mkSZNk9uzZUqdOHdf9+rPWYanjx4/nex70q7fz5HzsQsfojGs+EXunQx0HDx40qzX005Je5s6dK++++665rp+OODfW0NUELVq08LivefPmZgWO+8/2Qn+/9KueX3e6WkdXI/hy/uBJVz1pr8Utt9xihgFvv/12efzxx80KOMW5CeNgoV2Kl112mRkrc/+UoLe7du1qadvsRCf6aqiYMGGCzJo1K0/Xnp6DkiVLepwHHUPUP6DO86Bf16xZ4/GLqJ+y9Y3J+cdXj3F/DucxnMv89enTx/xc9ROX86KfkrVL13mdc2MNHS7MvSxbx/Tr1atnruvvkb65uP9ctXtex+fdz42GQg2QTvo7qH/ndLzfecy8efPMXBr3c9O0aVNbdrX7w8mTJ81cCHf6IVV/ropzUwQOmyw31Zm6o0ePNrN077//frPc1H2GO4rmwQcfNMuu5syZ49i3b5/rcvLkSY8ljboEddasWWZJY9euXc0l95LGfv36mSWrukyxatWqXpc0PvXUU2blwgcffMCSxkJwXxWiODfWLf+NiooySxs3b97sGDNmjPkZfv311x5LGvXv1cSJEx2rV692XH/99V6XNLZr184sWZ0/f75Z/eO+pFFXK+iSxttvv90sadS/ifo6dl7SWFRDhgxx1K5d27Xc9McffzRLrHX1kxPnpnBsESyUrqnXP5xaz0KXn+qaYviPZlBvF61t4aS/bP/4xz/M0ir9xbnhhhtM+HC3Y8cOx4ABA8w6bv0lfvLJJx2ZmZkex8yePdvRtm1bcy4bNmzo8RooXLDg3Fjnl19+MaFNP/w0a9bM8cknn3g8rssaX3zxRfPmo8f06dPHkZSU5HHMkSNHzJuV1lnQJcB33XWXWV7sTuss6NJWfQ59w9Q3ReQvNTXV/I7o+0apUqXM/+fnn3/eY1ko56Zw2DYdAAD4TcjPsQAAAMGDYAEAAPyGYAEAAPyGYAEAAPyGYAEAAPyGYAEAAPyGYAEAAPyGYAEAAPyGYAEAAPyGYAEAAPyGYAEAAPyGYAEAAMRf/j8goUp3h6C0OAAAAABJRU5ErkJggg==",
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "\n",
    "\n",
    "optimizer = torch.optim.Adam(model.parameters(), lr=1e-4, betas=(0.9, 0.98), eps=1e-9)\n",
    "loss_fn = torch.nn.CrossEntropyLoss(ignore_index=pad)\n",
    "\n",
    "\n",
    "def backward(logits, target):\n",
    "    loss = loss_fn(logits, target)\n",
    "    loss.backward()\n",
    "    optimizer.step()\n",
    "    optimizer.zero_grad()\n",
    "    return loss\n",
    "\n",
    "\n",
    "losses = train(model, train_loader, backward)\n",
    "\n",
    "plt.plot(losses)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "0c9bae10",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "source: 您 想要 茶 还是 咖啡 ？\n",
      "target: would you like tea or coffee ?\n",
      "output: would you like tea or coffee ?\n",
      "\n",
      "source: 那 是 一个 很 好 的 问题 。\n",
      "target: that 's a good question .\n",
      "output: that 's a good question .\n",
      "\n",
      "source: 这 不是 一件 开玩笑 的 事 。\n",
      "target: this is n't a joking matter .\n",
      "output: this is not something very important .\n",
      "\n",
      "source: 我 在 树林 里 迷路 了 。\n",
      "target: i got lost in the forest .\n",
      "output: i got lost in my way .\n",
      "\n",
      "source: 你 认为 如何 ?\n",
      "target: what do you think of that ?\n",
      "output: what do you think ?\n",
      "\n",
      "source: 去 纽约 旅行 吧 ！\n",
      "target: let 's take a trip to new york .\n",
      "output: let 's take a new car .\n",
      "\n",
      "source: 让 我们 去 野餐 。\n",
      "target: let 's go to the picnic .\n",
      "output: let 's go on a hurry .\n",
      "\n",
      "source: 你 看起来 就 像 你 的 母亲 。\n",
      "target: you look just like your mother .\n",
      "output: you look just like your mother .\n",
      "\n",
      "source: 你 来 这里 出差 的 ？\n",
      "target: are you here on business ?\n",
      "output: did you come here on your way ?\n",
      "\n",
      "source: 您 最 喜欢 什么 水果 ？\n",
      "target: what fruit do you like the best ?\n",
      "output: what do you like the best ?\n",
      "\n",
      "source: 我 没 听懂 您 的 意思 。\n",
      "target: i did n't understand you .\n",
      "output: i did n't mean to mean you .\n",
      "\n",
      "source: 今天 汤姆 有点 奇怪 。\n",
      "target: there 's something strange about tom today .\n",
      "output: tom is a strange today .\n",
      "\n",
      "source: 你 打算 怎么 回家 ？\n",
      "target: how are you going to get home ?\n",
      "output: how do you plan to go home ?\n",
      "\n",
      "source: 明天 我 得 购物 。\n",
      "target: tomorrow i have to go shopping .\n",
      "output: i have to be able to tomorrow .\n",
      "\n",
      "source: 她 骗 了 他 。\n",
      "target: she fooled him .\n",
      "output: she has gone .\n",
      "\n",
      "source: 我 运气 很 好 。\n",
      "target: i got lucky .\n",
      "output: i 'm very good at that .\n",
      "\n",
      "source: 他 几乎 每天 打电话 给 我 。\n",
      "target: he called me up almost every day .\n",
      "output: he called me up to call every day .\n",
      "\n",
      "source: 我 全身 酸痛 。\n",
      "target: i 'm sore all over .\n",
      "output: my sister is out .\n",
      "\n",
      "source: 是 你 杀 了 汤姆 吗 ？\n",
      "target: did you kill tom ?\n",
      "output: did you meet tom ?\n",
      "\n",
      "source: 我 都 没 考虑 过 。\n",
      "target: i did n't even consider that .\n",
      "output: i have n't thought about it .\n",
      "\n",
      "source: 请 关门 。\n",
      "target: close the door , please .\n",
      "output: please close the door .\n",
      "\n",
      "source: 让 我 给 你 一些 建议 。\n",
      "target: let me give you some advice .\n",
      "output: let me have some you some .\n",
      "\n",
      "source: 他 明天 将 打 棒球 。\n",
      "target: he plays baseball tomorrow .\n",
      "output: he will play baseball tomorrow .\n",
      "\n",
      "source: 你 想 怎么 做 就 怎么 做 。\n",
      "target: do as you like .\n",
      "output: do as you want .\n",
      "\n",
      "source: 我 保证 明天 就 做\n",
      "target: i promise i 'll do that tomorrow .\n",
      "output: i 'll do it tomorrow .\n",
      "\n",
      "source: 我 什么 经验 都 没有 。\n",
      "target: i do n't have any experience .\n",
      "output: i have no idea anything .\n",
      "\n",
      "source: 玛丽 似乎 对 这个 游戏 感到 无聊 。\n",
      "target: mary seems to be bored with the game .\n",
      "output: mary seems to be the same mistake .\n",
      "\n",
      "source: 她 只是 个 孩子 。\n",
      "target: she is nothing but a child .\n",
      "output: she is just a children .\n",
      "\n",
      "source: 我 在 找 你 。\n",
      "target: i 'm looking for you .\n",
      "output: i 'm looking for you .\n",
      "\n",
      "source: 比萨 是 我 最 喜欢 的 食物 。\n",
      "target: pizza is my favorite food .\n",
      "output: my favorite is my favorite .\n",
      "\n",
      "source: 我会 带 我 妹妹 去 派对 。\n",
      "target: i 'll bring my sister to the party .\n",
      "output: i 'll bring my sister to the party .\n",
      "\n",
      "source: 汤姆 买 了 新 笔记本 。\n",
      "target: tom bought a new notebook .\n",
      "output: tom bought a new car .\n",
      "\n"
     ]
    }
   ],
   "source": [
    "def print_result(source, target, output):\n",
    "    print(\"source:\", \" \".join(cmn_words[i] for i in source))\n",
    "    print(\"target:\", \" \".join(eng_words[i] for i in target))\n",
    "    print(\"output:\", \" \".join(eng_words[i] for i in output))\n",
    "    print()\n",
    "\n",
    "\n",
    "eval(model, test_loader, print_result, seq_len)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "base",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.13.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
